package com.yubest.webcollector;

import cn.edu.hfut.dmic.webcollector.model.CrawlDatum;
import cn.edu.hfut.dmic.webcollector.model.CrawlDatums;
import cn.edu.hfut.dmic.webcollector.model.Page;
import cn.edu.hfut.dmic.webcollector.plugin.ram.RamCrawler;
import org.jsoup.nodes.Element;
import org.jsoup.select.Elements;

/**
 * 爬虫-开源中国
 *
 * @Author: hweiyu
 * @Date: 2022/10/31 15:59
 */
public class OsChinaCrawler extends RamCrawler {

    public OsChinaCrawler(String url) {
        getConf().setDefaultUserAgent("Mozilla/5.0 (Windows NT 6.1; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/85.0.4183.83 Safari/537.36");
        getConf().setConnectTimeout(10000);
        getConf().setReadTimeout(20000);
        //失败后，重新请求时间间隔
        getConf().setWaitThreadEndTime(2000);
        //设置线程请求频数
        getConf().setExecuteInterval(1000);
        setThreads(1);

        addSeed(url, Type.TOPIC.name());
    }

    /**
     * 开源中国首页-技术问答模块
     * @param page
     * @param next
     */
    @Override
    public void visit(Page page, CrawlDatums next) {
        System.out.println("当前链接：" + page.url());
        if (page.matchType(Type.TOPIC.name())) {
            Elements elements = page.select(".panel-box--question-career .panel-content .panel-content__column");
            Element element = elements.get(0);
            Elements aList = element.select(".item-title");
            for (Element a : aList) {
                System.out.println(a.text() + " -> " + a.attr("href"));

                next.add(new CrawlDatum()
                        .url(a.attr("href"))
                        .type(Type.DETAIL.name()));
            }
        } else if (page.matchType(Type.DETAIL.name())) {
            Elements topic = page.select(".question-detail .article-title a");
            Elements context = page.select(".question-detail #articleContent");
            System.out.println("------" + topic.get(0).text() + "------ 开始 ------");
            System.out.println(context.get(0).text());
            System.out.println("------" + topic.get(0).text() + "------ 结束 ------");
        }
    }

    enum Type {
        TOPIC, DETAIL;
    }
}
